#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :get_zwjg.py
# @Time      :2023/11/8 
# @Author    :CL
# @email     :1037654919@qq.com
import re
import requests
from bs4 import BeautifulSoup
from utils import  mongo_manager,get_kuai_proxy
waijiaobu_zwjg = mongo_manager('waijiaobu_zwjg',db = 'waijiaobu')
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Referer": "https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743/",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "same-origin",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
    "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Linux\""
}
cookies = {
    "HMF_CI": "5a16d8bf0a3ab42047c26c3c3b5b60f423c865ec968964f521c79918fc4dac33f15dfed0818218fe0c7b40f92bff57dfba05612c7105ea346b813bfa47a317cb95",
    "_trs_uv": "lop388ic_469_fpck",
    "arialoadData": "true",
    "ariawapChangeViewPort": "false",
    "_trs_ua_s_1": "lopdklb6_469_7c9i",
    "HMY_JC": "41ca5c9c0e6a2ffda15d86cb0b22ef38bd702bdec33bbd6a39c674a1f970ece645,",
    "HBB_HC": "dfeec833c2433d6e57d399fe974888534d3a06a8f8d8cb31a564966b5d0913de288ab3fb24a2fb49d3db880ee567c17a8e"
}
# 驻外使馆
def get_zwsg(url = "https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743/yz_674745/"): #https://www.mfa.gov.cn/web/zwjg_674741/zwzlg_674757/yz_674759/
    response = requests.get(url, headers=headers,proxies=get_kuai_proxy(), timeout=10,cookies=cookies)
    print(response)
    response.encoding = 'utf-8'
    soups = BeautifulSoup(response.text, 'lxml')
    datas = soups.find('div', class_='bd').find_all('li')
    zwsg_dict ={}
    for data in datas:
        # print(data.get_text())
        # print(data.find('a').get('href'))
        zwsg_dict[data.get_text()] = data.find('a').get('href').replace('.', '')
    # print(zwjg_dict)
    return zwsg_dict
#驻外总领馆
def get_zwzlg(url = "https://www.mfa.gov.cn/web/zwjg_674741/zwzlg_674757/yz_674759/"): #
    response = requests.get(url, headers=headers,proxies=get_kuai_proxy(), timeout=10,cookies=cookies)
    print(response)
    response.encoding = 'utf-8'
    soups = BeautifulSoup(response.text, 'lxml')
    datas = soups.find('div', class_='bd').find_all('li')
    zwzlg_dict ={}
    for data in datas:
        # print(data.get_text())
        # print(data.find('a').get('href'))
        zwzlg_dict[data.get_text()] = data.find('a').get('href').replace('.', '')
    # print(zwzlg_dict)
    return zwzlg_dict
#驻外团、处
def get_zwtc(url = "https://www.mfa.gov.cn/web/zwjg_674741/zwtc_674771/"): #
    response = requests.get(url, headers=headers,proxies=get_kuai_proxy(), timeout=10,cookies=cookies)
    print(response)
    response.encoding = 'utf-8'
    soups = BeautifulSoup(response.text, 'lxml')
    datas = soups.find('div', class_='linkList2').find_all('li')
    zwtc_dict ={}
    for data in datas:
        zwtc_dict[data.get_text()] = data.find('a').get('href')
    return zwtc_dict
#  >驻外机构 list
def get_zwjg_list(url = 'https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743/yz_674745/'):
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743/yz_674745/",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-User": "?1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "HMF_CI": "5a16d8bf0a3ab42047c26c3c3b5b60f423c865ec968964f521c79918fc4dac33f15dfed0818218fe0c7b40f92bff57dfba05612c7105ea346b813bfa47a317cb95",
        "_trs_uv": "lop388ic_469_fpck",
        "arialoadData": "true",
        "ariawapChangeViewPort": "false",
        "_trs_ua_s_1": "lopdklb6_469_7c9i",
        "HMY_JC": "24af83986825cdb08a38fd45c3a4c1394afad47236571707d86083d809aad4df7d,",
        "HBB_HC": "43a3c657fd10f4f91fab9063c47fd35bc8c00bfff4532162f12f56755bddc90839e30195cc735ad383a8e31e966d627279"
    }
    response = requests.get(url, headers=headers, proxies=get_kuai_proxy(),timeout=10,cookies=cookies)
    response.encoding = 'utf-8'
    print(response)
    soups = BeautifulSoup(response.text, 'lxml')
    datas = soups.find('div', class_='rightbox').find_all('li')
    lists={}
    for data in datas:
        lists[data.get_text()] = data.find('a').get('href').replace('./', '')
    return lists

def get_zwjg_data( url = "https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743/yz_674745/200703/t20070313_7679799.shtml"):
    response = requests.get(url, headers=headers, proxies=get_kuai_proxy(),timeout=10,cookies=cookies)
    response.encoding='utf-8'
    print(response)
    soups = BeautifulSoup(response.text, 'lxml')
    datas = soups.find('div',class_='news-main').find_all('p')
    strs =''
    for data in datas:
        strs +=data.get_text() + '\n'
    return strs.strip()


if __name__ == "__main__":
    print()
    for continent,href in get_zwsg().items():
        url = 'https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743' + href
        if continent =='亚洲':
            url = 'https://www.mfa.gov.cn/web/zwjg_674741/zwsg_674743/yz_674745/'
        print(continent,url)
        lists = get_zwjg_list(url)
        for nation,href2 in lists.items():
            url2 = url + href2
            print(url2)
            try:
                data =get_zwjg_data(url2)
                result  ={'_id':nation,
                          'nation': nation,
                          'url': url2,
                          'continent': continent,
                          'type': 'zwjg',
                          'data':data}
                for p in data.split('\n'):
                    try:
                        infos = p.split('：',1)
                        if len(infos) ==2:
                            result[p.split('：')[0]] = p.split('：')[-1]
                    except:
                        pass
            except:
                result = {'_id': nation,
                          'nation': nation,
                          'url': url2,
                          'continent': continent,
                          'type': 'zwjg'}
            try:
                waijiaobu_zwjg.insertOne(result)
            except:
                waijiaobu_zwjg.updateOne({"_id":result["_id"]},result)
    for continent,href in get_zwzlg().items():
        url = 'https://www.mfa.gov.cn/web/zwjg_674741/zwzlg_674757' + href
        if continent =='亚洲':
            url = 'https://www.mfa.gov.cn/web/zwjg_674741/zwzlg_674757/yz_674745/'
        print(continent,url)
        lists = get_zwjg_list(url)
        for nation,href2 in lists.items():
            url2 = url + href2
            print(nation,url2)
            try:
                data =get_zwjg_data(url2)
                result  ={'_id':nation,
                          'nation': nation,
                          'url': url2,
                          'continent': continent,
                          'type': 'zwzlg',
                          'data':data}
                for p in data.split('\n'):
                    try:
                        infos = p.split('：',1)
                        if len(infos) ==2:
                            result[p.split('：')[0]] = p.split('：')[-1]
                    except:
                        pass
            except:
                result = {'_id': nation,
                          'nation': nation,
                          'url': url2,
                          'continent': continent,
                          'type': 'zwzlg'}
            try:
                waijiaobu_zwjg.insertOne(result)
            except:
                waijiaobu_zwjg.updateOne({"_id":result["_id"]},result)
    for nation, href2 in get_zwtc().items():
        url = 'https://www.mfa.gov.cn/web/zwjg_674741/zwtc_674771'
        url2 = url + href2
        print(nation,url2)
        try:
            data = get_zwjg_data(url2)
            result = {'_id': nation,
                      'nation': nation,
                      'url': url2,
                      'type': 'zwtc',
                      'data': data}
            for p in data.split('\n'):
                try:
                    infos = p.split('：', 1)
                    if len(infos) == 2:
                        result[p.split('：')[0]] = p.split('：')[-1]
                except:
                    pass
        except:
            result = {'_id': nation,
                      'nation': nation,
                      'url': url2,
                      'type': 'zwtc'}
        try:
            waijiaobu_zwjg.insertOne(result)
        except:
            waijiaobu_zwjg.updateOne({"_id": result["_id"]}, result)

